package sis.ra.founded;

import java.io.File;
import java.io.UnsupportedEncodingException;
import java.net.URLEncoder;
import java.util.ArrayList;
import java.util.HashMap;
import java.util.Iterator;


import sis.ra.partofspeech.wordindex;
import sis.ra.utility.Utils;
import sis.ra.yahoor.result.extractPattern;
import sis.ra.yahoor.result.parseYahooResults;
import sis.ra.yahoor.result.yahooresults;


public class PMI {

	static int alllines;
	static int compprod;
	
	public static String getYahooResultXML(String query, int resultnum)
	{
		try {
			query = URLEncoder.encode(query, "UTF-8");
		} catch (UnsupportedEncodingException e) {
			// TODO Auto-generated catch block
			e.printStackTrace();
		}
		String yahooquery="http://boss.yahooapis.com/ysearch/web/v1/"+query+"?format=xml&count="+resultnum+"&filter=-porn&appid=r.n3YhLV34FvfCRDPEJEm83gFUWbioKVrN3kcjeMol6J94rCnQndPPqzuOaIVA--";
		System.out.println(yahooquery);
		String content=Utils.fetchContentfrURL(yahooquery);
		System.out.println(content);
		return content;
	}
	
	public static void getYahooResult()
	{
		String path="C:\\Documents and Settings\\I820753\\Desktop\\dataset\\companylocation_training\\";
		String file=path+"CompanyLocation.txt";
		String content = Utils.readFile(file);
		String [] t=content.split("\n");
		for (int i=0;i<t.length;i++)
		{
			String line=t[i];
			String companyname=line.split(" ")[0];
			String location=line.split(" ")[1];
			if (line.contains("\" \""))
			{
				companyname=line.split("\" \"")[0].replace("\"", "");
				location=line.split("\" \"")[1].replace("\"", "");
			}
			if (line.contains("\" "))
			{
				companyname=line.split("\" ")[0].replace("\"", "");
				location=line.split("\" ")[1].replace("\"", "");
			}
			if (line.contains(" \""))
			{
				companyname=line.split(" \"")[0].replace("\"", "");
				location=line.split(" \"")[1].replace("\"", "");
			}
			String filename=(path+line).replace("\"", "").replace(",", "").replace("!", "").replace(".", "");
			System.out.println(filename+"\t"+companyname+"\t"+location);
			Utils.writeToFile(path+"companylocation.list", filename+"\t"+companyname+"\t"+location, true, true);
	//		String result=getYahooResultXML(line, 50);
	//		Utils.writeToFile(filename+".yahresult", result, true, true);
		}
	}
	
	public static void extractAbs(String xml, String filename, String company, String location)
	{
		
		String [] lines=xml.split("\n");
		String path="C:\\Documents and Settings\\I820753\\Desktop\\dataset\\companylocation_training\\";
		String line="";
		Utils.writeToFile(path+"companyproduct1.abs", filename, true, true);
		for (int i=0;i<lines.length;i++)
		{
			if (lines[i].contains("<abstract>"))
			{
				alllines=alllines+1;
			//	System.out.println(company+"\t"+location+"\t"+lines[i]);
				line=lines[i].replace("</b><b>", "");
				line=line.replace(company, "COMPANY");
				line=line.replace(location, "LOCATION");
				line=line.replace(company.replace(".", ""), "COMPANY");
				line=line.toLowerCase().replace(company.toLowerCase(), "COMPANY");
				line=line.toLowerCase().replace(location.toLowerCase(), "LOCATION");
				
				if (line.contains("COMPANY")&&line.contains("LOCATION"))
					compprod=compprod+1;
				else
				{
	//				System.out.println(company+"\t"+location+"\t"+line);
				}
				Utils.writeToFile(path+"companyproduct1.abs", company +"\t"+location+"\t"+line, true, true);
			}
		}
	}
	
	public static void matchCompanyLocation()
	{
		String path="C:\\Documents and Settings\\I820753\\Desktop\\dataset\\companylocation_training\\";
		File [] files=new File(path).listFiles();
		HashMap<String,String> companylocation=getCompanyLocation();
			
		for (int i=0;i<files.length;i++)
		{
			if (!files[i].toString().contains("yahresult")) continue;
			String xml=Utils.readFile(files[i].toString());
			String filename=files[i].toString().replace(".yahresult", "");
			filename=filename.replace("_training", "");
			String companyname=companylocation.get(filename).split("\t")[0];
			String location=companylocation.get(filename).split("\t")[1];
		
			extractAbs(xml,files[i].toString(), companyname, location);
//			ArrayList<yahooresults> rs=parseYahooResults.getYahooResult(xml);
//			for (int j=0;j<rs.size();j++)
//			{
//				System.out.println(rs.get(j).getAbs());
//			}
		}
		System.out.println(alllines+" "+compprod);
	}
	public static HashMap<String,String> getCompanyLocation()
	{
		HashMap<String,String> companyLocation = new HashMap<String,String> ();
		String companynamefile="C:\\Documents and Settings\\I820753\\Desktop\\dataset\\companylocation_training\\companylocation.list";
		String [] companylocation=Utils.readFile(companynamefile).split("\n");
		for (int i=0;i<companylocation.length;i++)
		{
			if (companylocation[i].length()<2) continue;
		System.out.println(companylocation[i]);
			String[] t=companylocation[i].split("\t");
			String filename=t[0];
			companyLocation.put(filename, t[1]+"\t"+t[2]);
		}
		return companyLocation;
	}
	
	public static void extractVerbPattern()
	{
		String path="C:\\Documents and Settings\\I820753\\Desktop\\dataset\\companylocation_training\\companyproduct1.abs";
		String [] lines=Utils.readFile(path).split("\n");
		String outfile="C:\\Documents and Settings\\I820753\\Desktop\\dataset\\companylocation_training\\verbs1.sent";
		int countline=0;
		int verbcount=0;
		int comploccount=0;
		for (int i=0;i<lines.length;i++)
		{
			if (lines[i].length()<2) continue;
			System.out.println(i+"\t"+lines[i]);
			boolean deal=false;
			if (lines[i].startsWith("C:\\Documents and Settings\\"))
			{
				Utils.writeToFile(outfile, lines[i], true, true);
				Utils.writeToFile(outfile, comploccount+"", true, true);
				comploccount=0;
			}
			if (lines[i].contains("company")&&lines[i].contains("location"))
			{
				if (lines[i].split("\t").length<3) continue;
				comploccount=comploccount+1;
				String line=lines[i].split("\t")[2];
				line=line.trim().replace("<abstract><![cdata[","").replace("]]></abstract>", "");
				String [] parts=line.split("<b>...</b>");
				
				for (int j=0;j<parts.length;j++)
				{	
					if (parts[j].trim().length()>2&&parts[j].contains("company")&&parts[j].contains("location"))
					{
						String thisline=parts[j].replace("<b>company</b>", "COMPANY").replace("<b>location</b>", "LOCATION");
						if (thisline.contains("COMPANY")&&thisline.contains("LOCATION"))
						{
							countline=countline+1;
							HashMap<Integer,wordindex> posresult=parseYahooResults.pos(thisline.trim(), "text");
							ArrayList<String> resultlist=parseYahooResults.getResultList (posresult);
							String result="";
							String verb="";
							String thisverb="";
							int companyverbdist=0;
							int verblocationdist=1000;
							int companypos=thisline.trim().indexOf("COMPANY");
							int locationpos=thisline.trim().indexOf("LOCATION");
							if (companypos>locationpos)
								companyverbdist=thisline.substring(locationpos, companypos).split(" ").length;
							else
								companyverbdist=thisline.substring(companypos,locationpos).split(" ").length;
							for (int n=0;n<resultlist.size()-1;n++)
							{
								if (resultlist.get(n).contains("V-")&&!resultlist.get(n+1).contains("V-"))
								{
									String thverb=resultlist.get(n).trim().split("\t")[1];
									System.out.println(thverb);
									int verbpos=thisline.trim().indexOf(thverb);
									if ((verbpos>companypos&&verbpos>locationpos)||(verbpos<companypos&&verbpos<locationpos)) continue;
									int thisverblocationdist=0;
									if (verbpos>companypos&&verbpos<locationpos)
									{
										thisverblocationdist=thisline.substring(verbpos, locationpos).split(" ").length;
									}else
									{
										thisverblocationdist=thisline.substring(locationpos, verbpos).split(" ").length;
									}
						//			verb=verb+"\t"+resultlist.get(n).trim();
									if (verblocationdist>thisverblocationdist)
									{
										verblocationdist=thisverblocationdist;
										thisverb=thverb;
									}
								}
								result=result+resultlist.get(n).trim();
							}
							if (resultlist.get(resultlist.size()-1).contains("V-"))
							{
								String thverb=resultlist.get(resultlist.size()-1).trim().split("\t")[1];
								if (thisline.indexOf(thverb)<thisline.trim().indexOf("LOCATION")) continue;
								int thisverblocationdist=thisline.substring(locationpos, thisline.trim().indexOf(thverb)).split(" ").length;
								if (verblocationdist>thisverblocationdist)
								{
									verblocationdist=thisverblocationdist;
									thisverb=thverb;
								}
								verbcount=verbcount+1;
							}
							result=result+resultlist.get(resultlist.size()-1).trim();
							if (thisverb.length()<2) continue;
							String stemverb=extractPattern.stem(thisverb);
							System.out.println(companyverbdist+"\t"+verblocationdist+"\t"+thisverb+"\t"+thisline.trim());
							Utils.writeToFile(outfile, "++>"+companyverbdist+"\t"+verblocationdist+"\t"+thisverb+"\t"+stemverb+"\t"+thisline.trim(), true, true);
							deal=true;
							verbcount=verbcount+1;
							
					//		System.out.println(result);
							if (verb.length()>2)
								System.out.println(verb.trim());
						}
					}					
				}
			}
			
			if (!deal)
			{
				if (lines[i].split("\t").length!=3) continue;
				String line=lines[i].split("\t")[2];
				line=line.trim().replace("<abstract><![cdata[","").replace("]]></abstract>", "");
				
				String [] parts=line.split("<b>...</b>");
				
				for (int j=0;j<parts.length;j++)
				{
					String pline=parts[j];
					HashMap<Integer,wordindex> posresult=parseYahooResults.pos(pline.trim(), "text");
					ArrayList<String> resultlist=parseYahooResults.getResultList (posresult);
					String result="";
					String verb="";
					for (int n=0;n<resultlist.size()-1;n++)
					{
						if (resultlist.get(n).contains("V-"))
						{
							String thverb=resultlist.get(n).trim().split("\t")[1].trim();
							String stemed=extractPattern.stem(thverb);
							Utils.writeToFile(outfile, "==>"+stemed+"\t"+thverb+"\t"+line.trim(), true, true);
						}
					}
				}
			}
		}// first for
		
		Utils.writeToFile(outfile, comploccount+"", true, true);
		System.out.println(countline+" "+verbcount);
	}
	public static void main(String[] args) {
//		getYahooResult();
//		matchCompanyLocation();
//		extractVerbPattern();
		
		String path="C:\\Documents and Settings\\I820753\\Desktop\\dataset\\companylocation_training\\verbs1.sent";
		String [] lines=Utils.readFile(path).split("\n");
		HashMap<String, Integer> companylocation=new HashMap<String, Integer>();
		HashMap<String, Integer> verb=new HashMap<String, Integer>();
		HashMap<String, Integer> verbcl=new HashMap<String, Integer>();
		String query="";
		String verbterm="";
		for (int i=0;i<lines.length;i++)
		{
			if (lines[i].length()<2) continue;
			if (lines[i].startsWith("C:\\Documents and Settings\\"))
			{
				query=lines[i];
			} else
			if (lines[i].startsWith("==>"))
			{
				verbterm=lines[i].substring(3).split("\t")[0];
				if (verb.containsKey(query+"\t"+verbterm))
				{
					Integer cur=verb.get(query+"\t"+verbterm)+1;
					verb.remove(query+"\t"+verbterm);
					verb.put(query+"\t"+verbterm, cur);
				}else{
					verb.put(query+"\t"+verbterm, 1);
				}				
			} else
			if (lines[i].startsWith("++>"))
			{
				verbterm=lines[i].substring(3).split("\t")[3];
				if (verb.containsKey(query+"\t"+verbterm))
				{
					Integer cur=verb.get(query+"\t"+verbterm)+1;
					verb.remove(query+"\t"+verbterm);
					verb.put(query+"\t"+verbterm, cur);
				}else{
					verb.put(query+"\t"+verbterm, 1);
				}	
				
				if (verbcl.containsKey(query+"\t"+verbterm))
				{
					Integer curcl=verbcl.get(query+"\t"+verbterm)+1;
					verbcl.remove(query+"\t"+verbterm);
					verbcl.put(query+"\t"+verbterm, curcl);
				}else{
					verbcl.put(query+"\t"+verbterm, 1);
				}	
			}
		}
		
		
		
		path="C:\\Documents and Settings\\I820753\\Desktop\\dataset\\companylocation_training\\companyproduct1.abs";
		lines=Utils.readFile(path).split("\n");
		int countline=0;
		int verbcount=0;
		int comploccount=0;
		query="";
		for (int i=0;i<lines.length;i++)
		{
			if (lines[i].length()<2) continue;
			System.out.println(i+"\t"+lines[i]);
			boolean deal=false;
			if (lines[i].startsWith("C:\\Documents and Settings\\"))
			{
				if (query.length()>2)
				{
					companylocation.put(query, comploccount);
				}
				query=lines[i];
				comploccount=0;
			}
			if (lines[i].contains("company")&&lines[i].contains("location"))
			{
				if (lines[i].split("\t").length<3) continue;
				comploccount=comploccount+1;
			}
		}
		companylocation.put(query, comploccount);
		
		HashMap<String, String> pmi=new HashMap<String, String> ();
		Iterator<String> verbincl=verbcl.keySet().iterator();
		while (verbincl.hasNext())
		{
			String t=verbincl.next();
			String q=t.split("\t")[0];
			String v=t.split("\t")[1];
			int clcount=companylocation.get(q);
			int vcount=verb.get(t);
			int vclcount=verbcl.get(t);
			System.out.println(t+"\t"+vclcount+"\t"+clcount+"\t"+vcount);
			
			double pmixy=(double)vclcount/(clcount*vcount);
			
			if (pmi.containsKey(v))
			{
				String s=pmi.get(v);
				double newpmi=Double.parseDouble(s.split("\t")[0])+pmixy;
				int newcount=Integer.parseInt(s.split("\t")[1])+1;
				pmi.remove(v);
				pmi.put(v, newpmi+"\t"+newcount);
			}else
			{
				pmi.put(v, pmixy+"\t"+1);
			}
		}
		
		Iterator<String> ts=pmi.keySet().iterator();
		while (ts.hasNext())
		{
			String t=ts.next();
			String s=pmi.get(t);
			double newpmi=Double.parseDouble(s.split("\t")[0]);
			int newcount=Integer.parseInt(s.split("\t")[1]);
			double avgpmi=newpmi/newcount;
			System.out.println(t+"\t"+avgpmi);
		}
	}

}
